{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b9583f02",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/jack/llama.cpp\n",
      "Already up to date.\n"
     ]
    }
   ],
   "source": [
    "!git clone https://github.com/ggerganov/llama.cpp # Get the code for the first time use\n",
    "%cd ~/llama.cpp\n",
    "!git pull"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4f651103-a6c8-4b7d-9f0f-be0553b22acf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[30m\u001b[43m13B\u001b[m\u001b[m                                ggml-vocab-gpt-2.gguf.inp\n",
      "\u001b[1m\u001b[36m13B-v2\u001b[m\u001b[m                             ggml-vocab-gpt-2.gguf.out\n",
      "\u001b[30m\u001b[43m30B\u001b[m\u001b[m                                ggml-vocab-gpt-neox.gguf\n",
      "\u001b[30m\u001b[43m65B\u001b[m\u001b[m                                ggml-vocab-gpt2.gguf\n",
      "\u001b[1m\u001b[36m70B-v2\u001b[m\u001b[m                             ggml-vocab-llama-bpe.gguf\n",
      "\u001b[1m\u001b[36m70B-v3-instruct\u001b[m\u001b[m                    ggml-vocab-llama-bpe.gguf.inp\n",
      "\u001b[30m\u001b[43m7B\u001b[m\u001b[m                                 ggml-vocab-llama-bpe.gguf.out\n",
      "\u001b[1m\u001b[36m7B-v2\u001b[m\u001b[m                              ggml-vocab-llama-spm.gguf\n",
      "\u001b[1m\u001b[36m8B-v3-instruct\u001b[m\u001b[m                     ggml-vocab-llama-spm.gguf.inp\n",
      "ggml-vocab-aquila.gguf             ggml-vocab-llama-spm.gguf.out\n",
      "ggml-vocab-baichuan.gguf           ggml-vocab-mpt.gguf\n",
      "ggml-vocab-bert-bge.gguf           ggml-vocab-mpt.gguf.inp\n",
      "ggml-vocab-bert-bge.gguf.inp       ggml-vocab-mpt.gguf.out\n",
      "ggml-vocab-bert-bge.gguf.out       ggml-vocab-phi-3.gguf\n",
      "ggml-vocab-command-r.gguf          ggml-vocab-phi-3.gguf.inp\n",
      "ggml-vocab-command-r.gguf.inp      ggml-vocab-phi-3.gguf.out\n",
      "ggml-vocab-command-r.gguf.out      ggml-vocab-refact.gguf\n",
      "ggml-vocab-deepseek-coder.gguf     ggml-vocab-refact.gguf.inp\n",
      "ggml-vocab-deepseek-coder.gguf.inp ggml-vocab-refact.gguf.out\n",
      "ggml-vocab-deepseek-coder.gguf.out ggml-vocab-stablelm.gguf\n",
      "ggml-vocab-deepseek-llm.gguf       ggml-vocab-starcoder.gguf\n",
      "ggml-vocab-deepseek-llm.gguf.inp   ggml-vocab-starcoder.gguf.inp\n",
      "ggml-vocab-deepseek-llm.gguf.out   ggml-vocab-starcoder.gguf.out\n",
      "ggml-vocab-falcon.gguf             tokenizer.model\n",
      "ggml-vocab-falcon.gguf.inp         \u001b[31mtokenizer_checklist.chk\u001b[m\u001b[m\n",
      "ggml-vocab-falcon.gguf.out         \u001b[1m\u001b[36mtokenizers\u001b[m\u001b[m\n",
      "ggml-vocab-gpt-2.gguf\n"
     ]
    }
   ],
   "source": [
    "# Obtain the original LLaMA model weights and place them in ./models\n",
    "!ls ./models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4e63016a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/jack/llama.cpp/models\n",
      "Updated Git hooks.\n",
      "Git LFS initialized.\n",
      "Cloning into 'Llama-3-8B-GGUF'...\n",
      "remote: Enumerating objects: 44, done.\u001b[K\n",
      "remote: Counting objects: 100% (41/41), done.\u001b[K\n",
      "remote: Compressing objects: 100% (41/41), done.\u001b[K\n",
      "remote: Total 44 (delta 9), reused 0 (delta 0), pack-reused 3 (from 1)\u001b[K\n",
      "Unpacking objects: 100% (44/44), 2.25 MiB | 8.12 MiB/s, done.\n",
      "Filtering content: 100% (8/8), 9.46 GiB | 10.41 MiB/s, done.\n",
      "Cloning into 'Llama-3-70B-GGUF'...\n",
      "remote: Enumerating objects: 67, done.\u001b[K\n",
      "remote: Counting objects: 100% (64/64), done.\u001b[K\n",
      "remote: Compressing objects: 100% (64/64), done.\u001b[K\n",
      "remote: Total 67 (delta 4), reused 0 (delta 0), pack-reused 3 (from 1)\u001b[K\n",
      "Unpacking objects: 100% (67/67), 2.26 MiB | 6.16 MiB/s, done.\n",
      "Filtering content: 100% (40/40), 22.45 GiB | 914.00 KiB/s, done.\n"
     ]
    }
   ],
   "source": [
    "# Clone/Download the model files from Meta HF repo: https://huggingface.co/meta-llama. Or feel free to clone from my HF repo:\n",
    "%cd models\n",
    "# !brew install git-lfs # Get git-lfs to clone large files\n",
    "!git lfs install\n",
    "!git clone https://huggingface.co/JaaackXD/Llama-3-8B-GGUF\n",
    "!git clone https://huggingface.co/JaaackXD/Llama-3-70B-GGUF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "603a64eb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/jack/llama.cpp\n"
     ]
    }
   ],
   "source": [
    "!mkdir 8B-v3\n",
    "!mv Llama-3-8B-GGUF/*.gguf 8B-v3\n",
    "!mkdir 70B-v3\n",
    "!mv Llama-3-70B-GGUF/*.gguf 70B-v3\n",
    "%cd .."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "43e9d80e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build\n",
    "!make clean && make -j"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "3ca0e26e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:hf-to-gguf:Loading model: Meta-Llama-3-70B\n",
      "INFO:gguf.gguf_writer:gguf: This GGUF file is for Little Endian only\n",
      "INFO:hf-to-gguf:Set model parameters\n",
      "INFO:hf-to-gguf:gguf: context length = 8192\n",
      "INFO:hf-to-gguf:gguf: embedding length = 8192\n",
      "INFO:hf-to-gguf:gguf: feed forward length = 28672\n",
      "INFO:hf-to-gguf:gguf: head count = 64\n",
      "INFO:hf-to-gguf:gguf: key-value head count = 8\n",
      "INFO:hf-to-gguf:gguf: rope theta = 500000.0\n",
      "INFO:hf-to-gguf:gguf: rms norm epsilon = 1e-05\n",
      "INFO:hf-to-gguf:gguf: file type = 1\n",
      "INFO:hf-to-gguf:Set model tokenizer\n",
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n",
      "INFO:gguf.vocab:Adding 280147 merge(s).\n",
      "INFO:gguf.vocab:Setting special token type bos to 128000\n",
      "INFO:gguf.vocab:Setting special token type eos to 128001\n",
      "INFO:hf-to-gguf:Exporting model to 'models/70B-v3/ggml-model-f16.gguf'\n",
      "INFO:hf-to-gguf:gguf: loading model part 'model-00001-of-00030.safetensors'\n",
      "INFO:hf-to-gguf:token_embd.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.0.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.0.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.0.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.0.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.0.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.0.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.0.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.0.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.0.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.1.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.1.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.1.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.1.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.1.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:gguf: loading model part 'model-00002-of-00030.safetensors'\n",
      "INFO:hf-to-gguf:blk.1.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.1.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.1.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.1.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.2.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.2.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.2.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.2.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.2.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.2.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.2.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.2.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.2.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.3.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.3.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.3.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.3.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.3.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.3.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.3.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.3.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.3.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.4.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.4.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.4.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.4.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:gguf: loading model part 'model-00003-of-00030.safetensors'\n",
      "INFO:hf-to-gguf:blk.4.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.4.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.4.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.4.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.4.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.5.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.5.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.5.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.5.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.5.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.5.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.5.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.5.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.5.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.6.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.6.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.6.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.6.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.6.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.6.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.6.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.6.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.6.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.7.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.7.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.7.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:gguf: loading model part 'model-00004-of-00030.safetensors'\n",
      "INFO:hf-to-gguf:blk.7.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.7.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.7.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.7.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.7.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.7.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.8.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.8.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.8.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.8.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.8.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.8.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.8.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.8.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.8.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.9.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.9.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.9.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.9.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.9.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.9.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.9.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.9.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.9.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:gguf: loading model part 'model-00005-of-00030.safetensors'\n",
      "INFO:hf-to-gguf:blk.10.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.10.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.10.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.10.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.10.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.10.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.10.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.10.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.10.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.11.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.11.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.11.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.11.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.11.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.11.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.11.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.11.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.11.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.12.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.12.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.12.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.12.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.12.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.12.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:gguf: loading model part 'model-00006-of-00030.safetensors'\n",
      "INFO:hf-to-gguf:blk.12.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.12.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.12.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.13.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.13.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.13.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.13.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.13.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.13.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.13.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.13.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.13.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.14.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.14.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.14.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.14.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.14.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.14.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.14.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.14.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.14.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.15.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.15.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.15.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.15.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.15.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:gguf: loading model part 'model-00007-of-00030.safetensors'\n",
      "INFO:hf-to-gguf:blk.15.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.15.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.15.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.15.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.16.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.16.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.16.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.16.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.16.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.16.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.16.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.16.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.16.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.17.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.17.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.17.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.17.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.17.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.17.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.17.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.17.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.17.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.18.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.18.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.18.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.18.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:gguf: loading model part 'model-00008-of-00030.safetensors'\n",
      "INFO:hf-to-gguf:blk.18.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.18.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.18.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.18.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.18.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.19.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.19.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.19.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.19.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.19.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.19.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.19.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.19.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.19.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.20.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.20.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.20.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.20.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.20.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.20.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.20.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.20.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.20.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.21.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.21.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.21.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:gguf: loading model part 'model-00009-of-00030.safetensors'\n",
      "INFO:hf-to-gguf:blk.21.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.21.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.21.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.21.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.21.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.21.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.22.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.22.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.22.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.22.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.22.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.22.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.22.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.22.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.22.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.23.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.23.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.23.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.23.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.23.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.23.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.23.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.23.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.23.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:gguf: loading model part 'model-00010-of-00030.safetensors'\n",
      "INFO:hf-to-gguf:blk.24.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.24.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.24.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.24.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.24.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.24.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.24.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.24.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.24.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.25.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.25.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.25.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.25.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.25.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.25.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.25.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.25.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.25.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.26.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.26.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.26.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.26.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.26.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.26.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:gguf: loading model part 'model-00011-of-00030.safetensors'\n",
      "INFO:hf-to-gguf:blk.26.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.26.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.26.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.27.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.27.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.27.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.27.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.27.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.27.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.27.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.27.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.27.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.28.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.28.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.28.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.28.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.28.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.28.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.28.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.28.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.28.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.29.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.29.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.29.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.29.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.29.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:gguf: loading model part 'model-00012-of-00030.safetensors'\n",
      "INFO:hf-to-gguf:blk.29.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.29.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.29.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.29.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.30.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.30.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.30.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.30.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.30.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.30.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.30.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.30.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.30.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.31.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.31.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.31.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.31.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.31.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.31.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.31.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.31.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.31.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.32.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.32.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.32.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.32.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:gguf: loading model part 'model-00013-of-00030.safetensors'\n",
      "INFO:hf-to-gguf:blk.32.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.32.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.32.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.32.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.32.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.33.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.33.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.33.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.33.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.33.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.33.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.33.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.33.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.33.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.34.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.34.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.34.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.34.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.34.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.34.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.34.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.34.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.34.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.35.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.35.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.35.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:gguf: loading model part 'model-00014-of-00030.safetensors'\n",
      "INFO:hf-to-gguf:blk.35.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.35.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.35.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.35.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.35.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.35.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.36.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.36.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.36.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.36.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.36.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.36.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.36.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.36.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.36.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.37.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.37.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.37.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.37.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.37.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.37.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.37.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.37.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.37.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:gguf: loading model part 'model-00015-of-00030.safetensors'\n",
      "INFO:hf-to-gguf:blk.38.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.38.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.38.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.38.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.38.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.38.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.38.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.38.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.38.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.39.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.39.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.39.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.39.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.39.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.39.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.39.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.39.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.39.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.40.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.40.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.40.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.40.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.40.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.40.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:gguf: loading model part 'model-00016-of-00030.safetensors'\n",
      "INFO:hf-to-gguf:blk.40.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.40.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.40.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.41.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.41.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.41.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.41.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.41.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.41.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.41.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.41.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.41.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.42.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.42.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.42.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.42.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.42.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.42.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.42.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.42.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.42.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.43.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.43.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.43.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.43.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.43.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:gguf: loading model part 'model-00017-of-00030.safetensors'\n",
      "INFO:hf-to-gguf:blk.43.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.43.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.43.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.43.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.44.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.44.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.44.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.44.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.44.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.44.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.44.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.44.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.44.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.45.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.45.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.45.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.45.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.45.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.45.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.45.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.45.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.45.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.46.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.46.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.46.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.46.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:gguf: loading model part 'model-00018-of-00030.safetensors'\n",
      "INFO:hf-to-gguf:blk.46.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.46.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.46.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.46.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.46.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.47.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.47.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.47.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.47.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.47.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.47.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.47.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.47.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.47.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.48.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.48.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.48.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.48.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.48.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.48.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.48.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.48.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.48.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.49.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.49.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.49.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:gguf: loading model part 'model-00019-of-00030.safetensors'\n",
      "INFO:hf-to-gguf:blk.49.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.49.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.49.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.49.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.49.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.49.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.50.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.50.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.50.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.50.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.50.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.50.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.50.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.50.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.50.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.51.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.51.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.51.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.51.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.51.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.51.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.51.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.51.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.51.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:gguf: loading model part 'model-00020-of-00030.safetensors'\n",
      "INFO:hf-to-gguf:blk.52.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.52.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.52.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.52.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.52.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.52.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.52.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.52.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.52.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.53.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.53.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.53.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.53.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.53.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.53.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.53.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.53.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.53.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.54.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.54.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.54.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.54.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.54.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.54.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:gguf: loading model part 'model-00021-of-00030.safetensors'\n",
      "INFO:hf-to-gguf:blk.54.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.54.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.54.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.55.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.55.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.55.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.55.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.55.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.55.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.55.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.55.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.55.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.56.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.56.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.56.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.56.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.56.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.56.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.56.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.56.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.56.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.57.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.57.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.57.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.57.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.57.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:gguf: loading model part 'model-00022-of-00030.safetensors'\n",
      "INFO:hf-to-gguf:blk.57.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.57.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.57.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.57.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.58.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.58.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.58.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.58.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.58.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.58.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.58.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.58.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.58.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.59.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.59.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.59.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.59.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.59.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.59.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.59.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.59.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.59.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.60.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.60.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.60.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.60.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:gguf: loading model part 'model-00023-of-00030.safetensors'\n",
      "INFO:hf-to-gguf:blk.60.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.60.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.60.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.60.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.60.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.61.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.61.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.61.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.61.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.61.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.61.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.61.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.61.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.61.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.62.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.62.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.62.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.62.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.62.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.62.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.62.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.62.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.62.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.63.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.63.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.63.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:gguf: loading model part 'model-00024-of-00030.safetensors'\n",
      "INFO:hf-to-gguf:blk.63.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.63.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.63.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.63.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.63.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.63.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.64.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.64.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.64.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.64.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.64.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.64.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.64.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.64.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.64.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.65.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.65.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.65.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.65.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.65.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.65.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.65.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.65.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.65.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:gguf: loading model part 'model-00025-of-00030.safetensors'\n",
      "INFO:hf-to-gguf:blk.66.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.66.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.66.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.66.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.66.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.66.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.66.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.66.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.66.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.67.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.67.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.67.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.67.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.67.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.67.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.67.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.67.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.67.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.68.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.68.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.68.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.68.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.68.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.68.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:gguf: loading model part 'model-00026-of-00030.safetensors'\n",
      "INFO:hf-to-gguf:blk.68.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.68.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.68.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.69.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.69.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.69.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.69.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.69.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.69.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.69.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.69.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.69.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.70.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.70.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.70.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.70.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.70.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.70.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.70.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.70.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.70.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.71.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.71.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.71.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.71.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.71.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:gguf: loading model part 'model-00027-of-00030.safetensors'\n",
      "INFO:hf-to-gguf:blk.71.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.71.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.71.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.71.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.72.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.72.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.72.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.72.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.72.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.72.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.72.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.72.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.72.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.73.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.73.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.73.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.73.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.73.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.73.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.73.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.73.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.73.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.74.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.74.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.74.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.74.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:gguf: loading model part 'model-00028-of-00030.safetensors'\n",
      "INFO:hf-to-gguf:blk.74.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.74.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.74.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.74.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.74.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.75.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.75.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.75.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.75.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.75.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.75.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.75.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.75.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.75.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.76.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.76.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.76.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.76.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.76.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.76.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.76.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.76.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.76.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.77.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.77.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.77.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:gguf: loading model part 'model-00029-of-00030.safetensors'\n",
      "INFO:hf-to-gguf:blk.77.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.77.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.77.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.77.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.77.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.77.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.78.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.78.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.78.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.78.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.78.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.78.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.78.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.78.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.78.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.79.attn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.79.ffn_down.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.79.ffn_gate.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.79.ffn_up.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.79.ffn_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:blk.79.attn_k.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.79.attn_output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.79.attn_q.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:blk.79.attn_v.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:output_norm.weight, n_dims = 1, torch.bfloat16 --> float32\n",
      "INFO:hf-to-gguf:gguf: loading model part 'model-00030-of-00030.safetensors'\n",
      "INFO:hf-to-gguf:output.weight, n_dims = 2, torch.bfloat16 --> float16\n",
      "INFO:hf-to-gguf:Model successfully exported to 'models/70B-v3/ggml-model-f16.gguf'\n",
      "main: build = 2778 (3ea0d360)\n",
      "main: built with Apple clang version 15.0.0 (clang-1500.3.9.4) for arm64-apple-darwin23.4.0\n",
      "main: quantizing 'models/70B-v3/ggml-model-f16.gguf' to 'models/70B-v3/ggml-model-Q4_K_M.gguf' as Q4_K_M\n",
      "llama_model_loader: loaded meta data with 20 key-value pairs and 723 tensors from models/70B-v3/ggml-model-f16.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = Meta-Llama-3-70B\n",
      "llama_model_loader: - kv   2:                          llama.block_count u32              = 80\n",
      "llama_model_loader: - kv   3:                       llama.context_length u32              = 8192\n",
      "llama_model_loader: - kv   4:                     llama.embedding_length u32              = 8192\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 28672\n",
      "llama_model_loader: - kv   6:                 llama.attention.head_count u32              = 64\n",
      "llama_model_loader: - kv   7:              llama.attention.head_count_kv u32              = 8\n",
      "llama_model_loader: - kv   8:                       llama.rope.freq_base f32              = 500000.000000\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 1\n",
      "llama_model_loader: - kv  11:                           llama.vocab_size u32              = 128256\n",
      "llama_model_loader: - kv  12:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv  13:                       tokenizer.ggml.model str              = gpt2\n",
      "llama_model_loader: - kv  14:                         tokenizer.ggml.pre str              = llama-bpe\n",
      "llama_model_loader: - kv  15:                      tokenizer.ggml.tokens arr[str,128256]  = [\"!\", \"\\\"\", \"#\", \"$\", \"%\", \"&\", \"'\", ...\n",
      "llama_model_loader: - kv  16:                  tokenizer.ggml.token_type arr[i32,128256]  = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...\n",
      "llama_model_loader: - kv  17:                      tokenizer.ggml.merges arr[str,280147]  = [\"Ġ Ġ\", \"Ġ ĠĠĠ\", \"ĠĠ ĠĠ\", \"...\n",
      "llama_model_loader: - kv  18:                tokenizer.ggml.bos_token_id u32              = 128000\n",
      "llama_model_loader: - kv  19:                tokenizer.ggml.eos_token_id u32              = 128001\n",
      "llama_model_loader: - type  f32:  161 tensors\n",
      "llama_model_loader: - type  f16:  562 tensors\n",
      "[   1/ 723]                    token_embd.weight - [ 8192, 128256,     1,     1], type =    f16, converting to q4_K .. size =  2004.00 MiB ->   563.62 MiB\n",
      "[   2/ 723]               blk.0.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[   3/ 723]                blk.0.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q6_K .. size =   448.00 MiB ->   183.75 MiB\n",
      "[   4/ 723]                blk.0.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[   5/ 723]                  blk.0.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[   6/ 723]                blk.0.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[   7/ 723]                  blk.0.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[   8/ 723]             blk.0.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[   9/ 723]                  blk.0.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[  10/ 723]                  blk.0.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q6_K .. size =    16.00 MiB ->     6.56 MiB\n",
      "[  11/ 723]                blk.1.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[  12/ 723]                  blk.1.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[  13/ 723]             blk.1.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[  14/ 723]                  blk.1.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[  15/ 723]                  blk.1.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q6_K .. size =    16.00 MiB ->     6.56 MiB\n",
      "[  16/ 723]               blk.1.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  17/ 723]                blk.1.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q6_K .. size =   448.00 MiB ->   183.75 MiB\n",
      "[  18/ 723]                  blk.1.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[  19/ 723]                blk.1.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  20/ 723]               blk.2.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  21/ 723]                blk.2.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q6_K .. size =   448.00 MiB ->   183.75 MiB\n",
      "[  22/ 723]                blk.2.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[  23/ 723]                  blk.2.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[  24/ 723]                blk.2.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  25/ 723]                  blk.2.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[  26/ 723]             blk.2.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[  27/ 723]                  blk.2.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[  28/ 723]                  blk.2.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q6_K .. size =    16.00 MiB ->     6.56 MiB\n",
      "[  29/ 723]               blk.3.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  30/ 723]                blk.3.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q6_K .. size =   448.00 MiB ->   183.75 MiB\n",
      "[  31/ 723]                blk.3.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[  32/ 723]                  blk.3.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[  33/ 723]                blk.3.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  34/ 723]                  blk.3.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[  35/ 723]             blk.3.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[  36/ 723]                  blk.3.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[  37/ 723]                  blk.3.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q6_K .. size =    16.00 MiB ->     6.56 MiB\n",
      "[  38/ 723]                  blk.4.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[  39/ 723]             blk.4.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[  40/ 723]                  blk.4.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[  41/ 723]                  blk.4.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q6_K .. size =    16.00 MiB ->     6.56 MiB\n",
      "[  42/ 723]               blk.4.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  43/ 723]                blk.4.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q6_K .. size =   448.00 MiB ->   183.75 MiB\n",
      "[  44/ 723]                blk.4.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[  45/ 723]                  blk.4.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[  46/ 723]                blk.4.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  47/ 723]               blk.5.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  48/ 723]                blk.5.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q6_K .. size =   448.00 MiB ->   183.75 MiB\n",
      "[  49/ 723]                blk.5.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[  50/ 723]                  blk.5.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[  51/ 723]                blk.5.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  52/ 723]                  blk.5.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[  53/ 723]             blk.5.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[  54/ 723]                  blk.5.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[  55/ 723]                  blk.5.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q6_K .. size =    16.00 MiB ->     6.56 MiB\n",
      "[  56/ 723]               blk.6.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  57/ 723]                blk.6.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q6_K .. size =   448.00 MiB ->   183.75 MiB\n",
      "[  58/ 723]                blk.6.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[  59/ 723]                  blk.6.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[  60/ 723]                blk.6.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  61/ 723]                  blk.6.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[  62/ 723]             blk.6.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[  63/ 723]                  blk.6.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[  64/ 723]                  blk.6.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q6_K .. size =    16.00 MiB ->     6.56 MiB\n",
      "[  65/ 723]                  blk.7.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[  66/ 723]                  blk.7.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[  67/ 723]                  blk.7.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q6_K .. size =    16.00 MiB ->     6.56 MiB\n",
      "[  68/ 723]               blk.7.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  69/ 723]                blk.7.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q6_K .. size =   448.00 MiB ->   183.75 MiB\n",
      "[  70/ 723]                blk.7.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[  71/ 723]                  blk.7.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[  72/ 723]                blk.7.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  73/ 723]             blk.7.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[  74/ 723]               blk.8.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  75/ 723]                blk.8.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q6_K .. size =   448.00 MiB ->   183.75 MiB\n",
      "[  76/ 723]                blk.8.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[  77/ 723]                  blk.8.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[  78/ 723]                blk.8.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  79/ 723]                  blk.8.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[  80/ 723]             blk.8.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[  81/ 723]                  blk.8.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[  82/ 723]                  blk.8.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q6_K .. size =    16.00 MiB ->     6.56 MiB\n",
      "[  83/ 723]               blk.9.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  84/ 723]                blk.9.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q6_K .. size =   448.00 MiB ->   183.75 MiB\n",
      "[  85/ 723]                blk.9.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[  86/ 723]                  blk.9.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[  87/ 723]                blk.9.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  88/ 723]                  blk.9.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[  89/ 723]             blk.9.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[  90/ 723]                  blk.9.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[  91/ 723]                  blk.9.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q6_K .. size =    16.00 MiB ->     6.56 MiB\n",
      "[  92/ 723]              blk.10.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  93/ 723]               blk.10.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[  94/ 723]               blk.10.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[  95/ 723]                 blk.10.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[  96/ 723]               blk.10.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  97/ 723]                 blk.10.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[  98/ 723]            blk.10.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[  99/ 723]                 blk.10.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 100/ 723]                 blk.10.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q5_K .. size =    16.00 MiB ->     5.50 MiB\n",
      "[ 101/ 723]              blk.11.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 102/ 723]               blk.11.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 103/ 723]               blk.11.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 104/ 723]                 blk.11.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 105/ 723]               blk.11.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 106/ 723]                 blk.11.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 107/ 723]            blk.11.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 108/ 723]                 blk.11.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 109/ 723]                 blk.11.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q5_K .. size =    16.00 MiB ->     5.50 MiB\n",
      "[ 110/ 723]               blk.12.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 111/ 723]                 blk.12.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 112/ 723]                 blk.12.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 113/ 723]            blk.12.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 114/ 723]                 blk.12.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 115/ 723]                 blk.12.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q6_K .. size =    16.00 MiB ->     6.56 MiB\n",
      "[ 116/ 723]              blk.12.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 117/ 723]               blk.12.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q6_K .. size =   448.00 MiB ->   183.75 MiB\n",
      "[ 118/ 723]               blk.12.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 119/ 723]              blk.13.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 120/ 723]               blk.13.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 121/ 723]               blk.13.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 122/ 723]                 blk.13.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 123/ 723]               blk.13.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 124/ 723]                 blk.13.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 125/ 723]            blk.13.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 126/ 723]                 blk.13.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 127/ 723]                 blk.13.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q5_K .. size =    16.00 MiB ->     5.50 MiB\n",
      "[ 128/ 723]              blk.14.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 129/ 723]               blk.14.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 130/ 723]               blk.14.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 131/ 723]                 blk.14.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 132/ 723]               blk.14.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 133/ 723]                 blk.14.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 134/ 723]            blk.14.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 135/ 723]                 blk.14.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 136/ 723]                 blk.14.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q5_K .. size =    16.00 MiB ->     5.50 MiB\n",
      "[ 137/ 723]               blk.15.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 138/ 723]                 blk.15.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 139/ 723]            blk.15.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 140/ 723]                 blk.15.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 141/ 723]                 blk.15.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q6_K .. size =    16.00 MiB ->     6.56 MiB\n",
      "[ 142/ 723]              blk.15.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 143/ 723]               blk.15.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q6_K .. size =   448.00 MiB ->   183.75 MiB\n",
      "[ 144/ 723]                 blk.15.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 145/ 723]               blk.15.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 146/ 723]              blk.16.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 147/ 723]               blk.16.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 148/ 723]               blk.16.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 149/ 723]                 blk.16.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 150/ 723]               blk.16.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 151/ 723]                 blk.16.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 152/ 723]            blk.16.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 153/ 723]                 blk.16.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 154/ 723]                 blk.16.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q5_K .. size =    16.00 MiB ->     5.50 MiB\n",
      "[ 155/ 723]              blk.17.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 156/ 723]               blk.17.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 157/ 723]               blk.17.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 158/ 723]                 blk.17.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 159/ 723]               blk.17.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 160/ 723]                 blk.17.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 161/ 723]            blk.17.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 162/ 723]                 blk.17.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 163/ 723]                 blk.17.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q5_K .. size =    16.00 MiB ->     5.50 MiB\n",
      "[ 164/ 723]                 blk.18.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 165/ 723]            blk.18.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 166/ 723]                 blk.18.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 167/ 723]                 blk.18.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q6_K .. size =    16.00 MiB ->     6.56 MiB\n",
      "[ 168/ 723]              blk.18.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 169/ 723]               blk.18.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q6_K .. size =   448.00 MiB ->   183.75 MiB\n",
      "[ 170/ 723]               blk.18.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 171/ 723]                 blk.18.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 172/ 723]               blk.18.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 173/ 723]              blk.19.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 174/ 723]               blk.19.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 175/ 723]               blk.19.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 176/ 723]                 blk.19.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 177/ 723]               blk.19.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 178/ 723]                 blk.19.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 179/ 723]            blk.19.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 180/ 723]                 blk.19.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 181/ 723]                 blk.19.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q5_K .. size =    16.00 MiB ->     5.50 MiB\n",
      "[ 182/ 723]              blk.20.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 183/ 723]               blk.20.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 184/ 723]               blk.20.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 185/ 723]                 blk.20.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 186/ 723]               blk.20.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 187/ 723]                 blk.20.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 188/ 723]            blk.20.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 189/ 723]                 blk.20.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 190/ 723]                 blk.20.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q5_K .. size =    16.00 MiB ->     5.50 MiB\n",
      "[ 191/ 723]                 blk.21.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 192/ 723]                 blk.21.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 193/ 723]                 blk.21.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q6_K .. size =    16.00 MiB ->     6.56 MiB\n",
      "[ 194/ 723]              blk.21.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 195/ 723]               blk.21.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q6_K .. size =   448.00 MiB ->   183.75 MiB\n",
      "[ 196/ 723]               blk.21.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 197/ 723]                 blk.21.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 198/ 723]               blk.21.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 199/ 723]            blk.21.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 200/ 723]              blk.22.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 201/ 723]               blk.22.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 202/ 723]               blk.22.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 203/ 723]                 blk.22.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 204/ 723]               blk.22.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 205/ 723]                 blk.22.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 206/ 723]            blk.22.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 207/ 723]                 blk.22.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 208/ 723]                 blk.22.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q5_K .. size =    16.00 MiB ->     5.50 MiB\n",
      "[ 209/ 723]              blk.23.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 210/ 723]               blk.23.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 211/ 723]               blk.23.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 212/ 723]                 blk.23.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 213/ 723]               blk.23.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 214/ 723]                 blk.23.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 215/ 723]            blk.23.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 216/ 723]                 blk.23.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 217/ 723]                 blk.23.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q5_K .. size =    16.00 MiB ->     5.50 MiB\n",
      "[ 218/ 723]              blk.24.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 219/ 723]               blk.24.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q6_K .. size =   448.00 MiB ->   183.75 MiB\n",
      "[ 220/ 723]               blk.24.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 221/ 723]                 blk.24.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 222/ 723]               blk.24.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 223/ 723]                 blk.24.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 224/ 723]            blk.24.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 225/ 723]                 blk.24.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 226/ 723]                 blk.24.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q6_K .. size =    16.00 MiB ->     6.56 MiB\n",
      "[ 227/ 723]              blk.25.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 228/ 723]               blk.25.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 229/ 723]               blk.25.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 230/ 723]                 blk.25.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 231/ 723]               blk.25.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 232/ 723]                 blk.25.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 233/ 723]            blk.25.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 234/ 723]                 blk.25.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 235/ 723]                 blk.25.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q5_K .. size =    16.00 MiB ->     5.50 MiB\n",
      "[ 236/ 723]               blk.26.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 237/ 723]                 blk.26.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 238/ 723]                 blk.26.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 239/ 723]            blk.26.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 240/ 723]                 blk.26.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 241/ 723]                 blk.26.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q5_K .. size =    16.00 MiB ->     5.50 MiB\n",
      "[ 242/ 723]              blk.26.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 243/ 723]               blk.26.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 244/ 723]               blk.26.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 245/ 723]              blk.27.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 246/ 723]               blk.27.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q6_K .. size =   448.00 MiB ->   183.75 MiB\n",
      "[ 247/ 723]               blk.27.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 248/ 723]                 blk.27.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 249/ 723]               blk.27.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 250/ 723]                 blk.27.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 251/ 723]            blk.27.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 252/ 723]                 blk.27.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 253/ 723]                 blk.27.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q6_K .. size =    16.00 MiB ->     6.56 MiB\n",
      "[ 254/ 723]              blk.28.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 255/ 723]               blk.28.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 256/ 723]               blk.28.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 257/ 723]                 blk.28.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 258/ 723]               blk.28.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 259/ 723]                 blk.28.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 260/ 723]            blk.28.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 261/ 723]                 blk.28.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 262/ 723]                 blk.28.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q5_K .. size =    16.00 MiB ->     5.50 MiB\n",
      "[ 263/ 723]               blk.29.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 264/ 723]                 blk.29.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 265/ 723]            blk.29.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 266/ 723]                 blk.29.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 267/ 723]                 blk.29.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q5_K .. size =    16.00 MiB ->     5.50 MiB\n",
      "[ 268/ 723]              blk.29.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 269/ 723]               blk.29.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 270/ 723]                 blk.29.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 271/ 723]               blk.29.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 272/ 723]              blk.30.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 273/ 723]               blk.30.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q6_K .. size =   448.00 MiB ->   183.75 MiB\n",
      "[ 274/ 723]               blk.30.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 275/ 723]                 blk.30.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 276/ 723]               blk.30.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 277/ 723]                 blk.30.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 278/ 723]            blk.30.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 279/ 723]                 blk.30.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 280/ 723]                 blk.30.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q6_K .. size =    16.00 MiB ->     6.56 MiB\n",
      "[ 281/ 723]              blk.31.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 282/ 723]               blk.31.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 283/ 723]               blk.31.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 284/ 723]                 blk.31.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 285/ 723]               blk.31.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 286/ 723]                 blk.31.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 287/ 723]            blk.31.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 288/ 723]                 blk.31.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 289/ 723]                 blk.31.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q5_K .. size =    16.00 MiB ->     5.50 MiB\n",
      "[ 290/ 723]                 blk.32.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 291/ 723]            blk.32.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 292/ 723]                 blk.32.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 293/ 723]                 blk.32.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q5_K .. size =    16.00 MiB ->     5.50 MiB\n",
      "[ 294/ 723]              blk.32.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 295/ 723]               blk.32.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 296/ 723]               blk.32.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 297/ 723]                 blk.32.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 298/ 723]               blk.32.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 299/ 723]              blk.33.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 300/ 723]               blk.33.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q6_K .. size =   448.00 MiB ->   183.75 MiB\n",
      "[ 301/ 723]               blk.33.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 302/ 723]                 blk.33.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 303/ 723]               blk.33.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 304/ 723]                 blk.33.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 305/ 723]            blk.33.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 306/ 723]                 blk.33.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 307/ 723]                 blk.33.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q6_K .. size =    16.00 MiB ->     6.56 MiB\n",
      "[ 308/ 723]              blk.34.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 309/ 723]               blk.34.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 310/ 723]               blk.34.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 311/ 723]                 blk.34.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 312/ 723]               blk.34.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 313/ 723]                 blk.34.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 314/ 723]            blk.34.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 315/ 723]                 blk.34.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 316/ 723]                 blk.34.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q5_K .. size =    16.00 MiB ->     5.50 MiB\n",
      "[ 317/ 723]                 blk.35.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 318/ 723]                 blk.35.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 319/ 723]                 blk.35.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q5_K .. size =    16.00 MiB ->     5.50 MiB\n",
      "[ 320/ 723]              blk.35.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 321/ 723]               blk.35.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 322/ 723]               blk.35.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 323/ 723]                 blk.35.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 324/ 723]               blk.35.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 325/ 723]            blk.35.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 326/ 723]              blk.36.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 327/ 723]               blk.36.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q6_K .. size =   448.00 MiB ->   183.75 MiB\n",
      "[ 328/ 723]               blk.36.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 329/ 723]                 blk.36.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 330/ 723]               blk.36.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 331/ 723]                 blk.36.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 332/ 723]            blk.36.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 333/ 723]                 blk.36.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 334/ 723]                 blk.36.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q6_K .. size =    16.00 MiB ->     6.56 MiB\n",
      "[ 335/ 723]              blk.37.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 336/ 723]               blk.37.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 337/ 723]               blk.37.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 338/ 723]                 blk.37.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 339/ 723]               blk.37.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 340/ 723]                 blk.37.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 341/ 723]            blk.37.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 342/ 723]                 blk.37.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 343/ 723]                 blk.37.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q5_K .. size =    16.00 MiB ->     5.50 MiB\n",
      "[ 344/ 723]              blk.38.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 345/ 723]               blk.38.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 346/ 723]               blk.38.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 347/ 723]                 blk.38.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 348/ 723]               blk.38.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 349/ 723]                 blk.38.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 350/ 723]            blk.38.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 351/ 723]                 blk.38.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 352/ 723]                 blk.38.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q5_K .. size =    16.00 MiB ->     5.50 MiB\n",
      "[ 353/ 723]              blk.39.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 354/ 723]               blk.39.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q6_K .. size =   448.00 MiB ->   183.75 MiB\n",
      "[ 355/ 723]               blk.39.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 356/ 723]                 blk.39.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 357/ 723]               blk.39.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 358/ 723]                 blk.39.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 359/ 723]            blk.39.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 360/ 723]                 blk.39.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 361/ 723]                 blk.39.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q6_K .. size =    16.00 MiB ->     6.56 MiB\n",
      "[ 362/ 723]               blk.40.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 363/ 723]                 blk.40.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 364/ 723]                 blk.40.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 365/ 723]            blk.40.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 366/ 723]                 blk.40.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 367/ 723]                 blk.40.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q5_K .. size =    16.00 MiB ->     5.50 MiB\n",
      "[ 368/ 723]              blk.40.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 369/ 723]               blk.40.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 370/ 723]               blk.40.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 371/ 723]              blk.41.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 372/ 723]               blk.41.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 373/ 723]               blk.41.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 374/ 723]                 blk.41.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 375/ 723]               blk.41.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 376/ 723]                 blk.41.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 377/ 723]            blk.41.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 378/ 723]                 blk.41.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 379/ 723]                 blk.41.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q5_K .. size =    16.00 MiB ->     5.50 MiB\n",
      "[ 380/ 723]              blk.42.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 381/ 723]               blk.42.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q6_K .. size =   448.00 MiB ->   183.75 MiB\n",
      "[ 382/ 723]               blk.42.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 383/ 723]                 blk.42.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 384/ 723]               blk.42.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 385/ 723]                 blk.42.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 386/ 723]            blk.42.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 387/ 723]                 blk.42.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 388/ 723]                 blk.42.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q6_K .. size =    16.00 MiB ->     6.56 MiB\n",
      "[ 389/ 723]               blk.43.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 390/ 723]                 blk.43.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 391/ 723]            blk.43.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 392/ 723]                 blk.43.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 393/ 723]                 blk.43.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q5_K .. size =    16.00 MiB ->     5.50 MiB\n",
      "[ 394/ 723]              blk.43.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 395/ 723]               blk.43.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 396/ 723]                 blk.43.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 397/ 723]               blk.43.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 398/ 723]              blk.44.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 399/ 723]               blk.44.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 400/ 723]               blk.44.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 401/ 723]                 blk.44.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 402/ 723]               blk.44.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 403/ 723]                 blk.44.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 404/ 723]            blk.44.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 405/ 723]                 blk.44.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 406/ 723]                 blk.44.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q5_K .. size =    16.00 MiB ->     5.50 MiB\n",
      "[ 407/ 723]              blk.45.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 408/ 723]               blk.45.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q6_K .. size =   448.00 MiB ->   183.75 MiB\n",
      "[ 409/ 723]               blk.45.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 410/ 723]                 blk.45.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 411/ 723]               blk.45.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 412/ 723]                 blk.45.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 413/ 723]            blk.45.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 414/ 723]                 blk.45.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 415/ 723]                 blk.45.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q6_K .. size =    16.00 MiB ->     6.56 MiB\n",
      "[ 416/ 723]                 blk.46.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 417/ 723]            blk.46.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 418/ 723]                 blk.46.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 419/ 723]                 blk.46.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q5_K .. size =    16.00 MiB ->     5.50 MiB\n",
      "[ 420/ 723]              blk.46.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 421/ 723]               blk.46.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 422/ 723]               blk.46.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 423/ 723]                 blk.46.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 424/ 723]               blk.46.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 425/ 723]              blk.47.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 426/ 723]               blk.47.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 427/ 723]               blk.47.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 428/ 723]                 blk.47.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 429/ 723]               blk.47.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 430/ 723]                 blk.47.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 431/ 723]            blk.47.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 432/ 723]                 blk.47.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 433/ 723]                 blk.47.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q5_K .. size =    16.00 MiB ->     5.50 MiB\n",
      "[ 434/ 723]              blk.48.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 435/ 723]               blk.48.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q6_K .. size =   448.00 MiB ->   183.75 MiB\n",
      "[ 436/ 723]               blk.48.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 437/ 723]                 blk.48.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 438/ 723]               blk.48.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 439/ 723]                 blk.48.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 440/ 723]            blk.48.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 441/ 723]                 blk.48.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 442/ 723]                 blk.48.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q6_K .. size =    16.00 MiB ->     6.56 MiB\n",
      "[ 443/ 723]                 blk.49.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 444/ 723]                 blk.49.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 445/ 723]                 blk.49.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q5_K .. size =    16.00 MiB ->     5.50 MiB\n",
      "[ 446/ 723]              blk.49.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 447/ 723]               blk.49.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 448/ 723]               blk.49.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 449/ 723]                 blk.49.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 450/ 723]               blk.49.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 451/ 723]            blk.49.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 452/ 723]              blk.50.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 453/ 723]               blk.50.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 454/ 723]               blk.50.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 455/ 723]                 blk.50.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 456/ 723]               blk.50.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 457/ 723]                 blk.50.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 458/ 723]            blk.50.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 459/ 723]                 blk.50.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 460/ 723]                 blk.50.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q5_K .. size =    16.00 MiB ->     5.50 MiB\n",
      "[ 461/ 723]              blk.51.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 462/ 723]               blk.51.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q6_K .. size =   448.00 MiB ->   183.75 MiB\n",
      "[ 463/ 723]               blk.51.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 464/ 723]                 blk.51.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 465/ 723]               blk.51.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 466/ 723]                 blk.51.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 467/ 723]            blk.51.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 468/ 723]                 blk.51.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 469/ 723]                 blk.51.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q6_K .. size =    16.00 MiB ->     6.56 MiB\n",
      "[ 470/ 723]              blk.52.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 471/ 723]               blk.52.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 472/ 723]               blk.52.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 473/ 723]                 blk.52.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 474/ 723]               blk.52.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 475/ 723]                 blk.52.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 476/ 723]            blk.52.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 477/ 723]                 blk.52.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 478/ 723]                 blk.52.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q5_K .. size =    16.00 MiB ->     5.50 MiB\n",
      "[ 479/ 723]              blk.53.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 480/ 723]               blk.53.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 481/ 723]               blk.53.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 482/ 723]                 blk.53.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 483/ 723]               blk.53.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 484/ 723]                 blk.53.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 485/ 723]            blk.53.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 486/ 723]                 blk.53.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 487/ 723]                 blk.53.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q5_K .. size =    16.00 MiB ->     5.50 MiB\n",
      "[ 488/ 723]               blk.54.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 489/ 723]                 blk.54.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 490/ 723]                 blk.54.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 491/ 723]            blk.54.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 492/ 723]                 blk.54.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 493/ 723]                 blk.54.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q6_K .. size =    16.00 MiB ->     6.56 MiB\n",
      "[ 494/ 723]              blk.54.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 495/ 723]               blk.54.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q6_K .. size =   448.00 MiB ->   183.75 MiB\n",
      "[ 496/ 723]               blk.54.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 497/ 723]              blk.55.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 498/ 723]               blk.55.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 499/ 723]               blk.55.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 500/ 723]                 blk.55.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 501/ 723]               blk.55.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 502/ 723]                 blk.55.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 503/ 723]            blk.55.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 504/ 723]                 blk.55.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 505/ 723]                 blk.55.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q5_K .. size =    16.00 MiB ->     5.50 MiB\n",
      "[ 506/ 723]              blk.56.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 507/ 723]               blk.56.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 508/ 723]               blk.56.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 509/ 723]                 blk.56.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 510/ 723]               blk.56.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 511/ 723]                 blk.56.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 512/ 723]            blk.56.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 513/ 723]                 blk.56.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 514/ 723]                 blk.56.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q5_K .. size =    16.00 MiB ->     5.50 MiB\n",
      "[ 515/ 723]               blk.57.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 516/ 723]                 blk.57.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 517/ 723]            blk.57.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 518/ 723]                 blk.57.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 519/ 723]                 blk.57.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q6_K .. size =    16.00 MiB ->     6.56 MiB\n",
      "[ 520/ 723]              blk.57.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 521/ 723]               blk.57.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q6_K .. size =   448.00 MiB ->   183.75 MiB\n",
      "[ 522/ 723]                 blk.57.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 523/ 723]               blk.57.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 524/ 723]              blk.58.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 525/ 723]               blk.58.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 526/ 723]               blk.58.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 527/ 723]                 blk.58.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 528/ 723]               blk.58.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 529/ 723]                 blk.58.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 530/ 723]            blk.58.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 531/ 723]                 blk.58.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 532/ 723]                 blk.58.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q5_K .. size =    16.00 MiB ->     5.50 MiB\n",
      "[ 533/ 723]              blk.59.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 534/ 723]               blk.59.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 535/ 723]               blk.59.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 536/ 723]                 blk.59.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 537/ 723]               blk.59.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 538/ 723]                 blk.59.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 539/ 723]            blk.59.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 540/ 723]                 blk.59.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 541/ 723]                 blk.59.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q5_K .. size =    16.00 MiB ->     5.50 MiB\n",
      "[ 542/ 723]                 blk.60.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 543/ 723]            blk.60.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 544/ 723]                 blk.60.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 545/ 723]                 blk.60.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q6_K .. size =    16.00 MiB ->     6.56 MiB\n",
      "[ 546/ 723]              blk.60.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 547/ 723]               blk.60.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q6_K .. size =   448.00 MiB ->   183.75 MiB\n",
      "[ 548/ 723]               blk.60.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 549/ 723]                 blk.60.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 550/ 723]               blk.60.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 551/ 723]              blk.61.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 552/ 723]               blk.61.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 553/ 723]               blk.61.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 554/ 723]                 blk.61.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 555/ 723]               blk.61.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 556/ 723]                 blk.61.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 557/ 723]            blk.61.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 558/ 723]                 blk.61.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 559/ 723]                 blk.61.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q5_K .. size =    16.00 MiB ->     5.50 MiB\n",
      "[ 560/ 723]              blk.62.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 561/ 723]               blk.62.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 562/ 723]               blk.62.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 563/ 723]                 blk.62.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 564/ 723]               blk.62.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 565/ 723]                 blk.62.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 566/ 723]            blk.62.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 567/ 723]                 blk.62.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 568/ 723]                 blk.62.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q5_K .. size =    16.00 MiB ->     5.50 MiB\n",
      "[ 569/ 723]                 blk.63.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 570/ 723]                 blk.63.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 571/ 723]                 blk.63.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q6_K .. size =    16.00 MiB ->     6.56 MiB\n",
      "[ 572/ 723]              blk.63.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 573/ 723]               blk.63.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q6_K .. size =   448.00 MiB ->   183.75 MiB\n",
      "[ 574/ 723]               blk.63.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 575/ 723]                 blk.63.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 576/ 723]               blk.63.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 577/ 723]            blk.63.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 578/ 723]              blk.64.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 579/ 723]               blk.64.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 580/ 723]               blk.64.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 581/ 723]                 blk.64.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 582/ 723]               blk.64.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 583/ 723]                 blk.64.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 584/ 723]            blk.64.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 585/ 723]                 blk.64.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 586/ 723]                 blk.64.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q5_K .. size =    16.00 MiB ->     5.50 MiB\n",
      "[ 587/ 723]              blk.65.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 588/ 723]               blk.65.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 589/ 723]               blk.65.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 590/ 723]                 blk.65.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 591/ 723]               blk.65.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 592/ 723]                 blk.65.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 593/ 723]            blk.65.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 594/ 723]                 blk.65.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 595/ 723]                 blk.65.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q5_K .. size =    16.00 MiB ->     5.50 MiB\n",
      "[ 596/ 723]              blk.66.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 597/ 723]               blk.66.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q6_K .. size =   448.00 MiB ->   183.75 MiB\n",
      "[ 598/ 723]               blk.66.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 599/ 723]                 blk.66.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 600/ 723]               blk.66.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 601/ 723]                 blk.66.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 602/ 723]            blk.66.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 603/ 723]                 blk.66.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 604/ 723]                 blk.66.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q6_K .. size =    16.00 MiB ->     6.56 MiB\n",
      "[ 605/ 723]              blk.67.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 606/ 723]               blk.67.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 607/ 723]               blk.67.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 608/ 723]                 blk.67.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 609/ 723]               blk.67.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 610/ 723]                 blk.67.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 611/ 723]            blk.67.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 612/ 723]                 blk.67.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 613/ 723]                 blk.67.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q5_K .. size =    16.00 MiB ->     5.50 MiB\n",
      "[ 614/ 723]               blk.68.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 615/ 723]                 blk.68.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 616/ 723]                 blk.68.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 617/ 723]            blk.68.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 618/ 723]                 blk.68.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 619/ 723]                 blk.68.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q5_K .. size =    16.00 MiB ->     5.50 MiB\n",
      "[ 620/ 723]              blk.68.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 621/ 723]               blk.68.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 622/ 723]               blk.68.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 623/ 723]              blk.69.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 624/ 723]               blk.69.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q6_K .. size =   448.00 MiB ->   183.75 MiB\n",
      "[ 625/ 723]               blk.69.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 626/ 723]                 blk.69.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 627/ 723]               blk.69.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 628/ 723]                 blk.69.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 629/ 723]            blk.69.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 630/ 723]                 blk.69.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 631/ 723]                 blk.69.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q6_K .. size =    16.00 MiB ->     6.56 MiB\n",
      "[ 632/ 723]              blk.70.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 633/ 723]               blk.70.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q6_K .. size =   448.00 MiB ->   183.75 MiB\n",
      "[ 634/ 723]               blk.70.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 635/ 723]                 blk.70.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 636/ 723]               blk.70.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 637/ 723]                 blk.70.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 638/ 723]            blk.70.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 639/ 723]                 blk.70.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 640/ 723]                 blk.70.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q6_K .. size =    16.00 MiB ->     6.56 MiB\n",
      "[ 641/ 723]               blk.71.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 642/ 723]                 blk.71.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 643/ 723]            blk.71.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 644/ 723]                 blk.71.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 645/ 723]                 blk.71.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q6_K .. size =    16.00 MiB ->     6.56 MiB\n",
      "[ 646/ 723]              blk.71.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 647/ 723]               blk.71.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q6_K .. size =   448.00 MiB ->   183.75 MiB\n",
      "[ 648/ 723]                 blk.71.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 649/ 723]               blk.71.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 650/ 723]              blk.72.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 651/ 723]               blk.72.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q6_K .. size =   448.00 MiB ->   183.75 MiB\n",
      "[ 652/ 723]               blk.72.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 653/ 723]                 blk.72.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 654/ 723]               blk.72.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 655/ 723]                 blk.72.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 656/ 723]            blk.72.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 657/ 723]                 blk.72.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 658/ 723]                 blk.72.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q6_K .. size =    16.00 MiB ->     6.56 MiB\n",
      "[ 659/ 723]              blk.73.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 660/ 723]               blk.73.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q6_K .. size =   448.00 MiB ->   183.75 MiB\n",
      "[ 661/ 723]               blk.73.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 662/ 723]                 blk.73.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 663/ 723]               blk.73.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 664/ 723]                 blk.73.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 665/ 723]            blk.73.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 666/ 723]                 blk.73.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 667/ 723]                 blk.73.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q6_K .. size =    16.00 MiB ->     6.56 MiB\n",
      "[ 668/ 723]                 blk.74.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 669/ 723]            blk.74.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 670/ 723]                 blk.74.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 671/ 723]                 blk.74.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q6_K .. size =    16.00 MiB ->     6.56 MiB\n",
      "[ 672/ 723]              blk.74.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 673/ 723]               blk.74.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q6_K .. size =   448.00 MiB ->   183.75 MiB\n",
      "[ 674/ 723]               blk.74.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 675/ 723]                 blk.74.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 676/ 723]               blk.74.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 677/ 723]              blk.75.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 678/ 723]               blk.75.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q6_K .. size =   448.00 MiB ->   183.75 MiB\n",
      "[ 679/ 723]               blk.75.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 680/ 723]                 blk.75.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 681/ 723]               blk.75.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 682/ 723]                 blk.75.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 683/ 723]            blk.75.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 684/ 723]                 blk.75.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 685/ 723]                 blk.75.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q6_K .. size =    16.00 MiB ->     6.56 MiB\n",
      "[ 686/ 723]              blk.76.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 687/ 723]               blk.76.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q6_K .. size =   448.00 MiB ->   183.75 MiB\n",
      "[ 688/ 723]               blk.76.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 689/ 723]                 blk.76.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 690/ 723]               blk.76.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 691/ 723]                 blk.76.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 692/ 723]            blk.76.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 693/ 723]                 blk.76.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 694/ 723]                 blk.76.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q6_K .. size =    16.00 MiB ->     6.56 MiB\n",
      "[ 695/ 723]                 blk.77.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 696/ 723]                 blk.77.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 697/ 723]                 blk.77.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q6_K .. size =    16.00 MiB ->     6.56 MiB\n",
      "[ 698/ 723]              blk.77.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 699/ 723]               blk.77.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q6_K .. size =   448.00 MiB ->   183.75 MiB\n",
      "[ 700/ 723]               blk.77.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 701/ 723]                 blk.77.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 702/ 723]               blk.77.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 703/ 723]            blk.77.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 704/ 723]              blk.78.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 705/ 723]               blk.78.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q6_K .. size =   448.00 MiB ->   183.75 MiB\n",
      "[ 706/ 723]               blk.78.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 707/ 723]                 blk.78.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 708/ 723]               blk.78.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 709/ 723]                 blk.78.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 710/ 723]            blk.78.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 711/ 723]                 blk.78.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 712/ 723]                 blk.78.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q6_K .. size =    16.00 MiB ->     6.56 MiB\n",
      "[ 713/ 723]              blk.79.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 714/ 723]               blk.79.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, converting to q6_K .. size =   448.00 MiB ->   183.75 MiB\n",
      "[ 715/ 723]               blk.79.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 716/ 723]                 blk.79.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, converting to q4_K .. size =   448.00 MiB ->   126.00 MiB\n",
      "[ 717/ 723]               blk.79.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 718/ 723]                 blk.79.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q4_K .. size =    16.00 MiB ->     4.50 MiB\n",
      "[ 719/ 723]            blk.79.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 720/ 723]                 blk.79.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, converting to q4_K .. size =   128.00 MiB ->    36.00 MiB\n",
      "[ 721/ 723]                 blk.79.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, converting to q6_K .. size =    16.00 MiB ->     6.56 MiB\n",
      "[ 722/ 723]                   output_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 723/ 723]                        output.weight - [ 8192, 128256,     1,     1], type =    f16, converting to q6_K .. size =  2004.00 MiB ->   821.95 MiB\n",
      "llama_model_quantize_internal: model size  = 134573.03 MB\n",
      "llama_model_quantize_internal: quant size  = 40543.11 MB\n",
      "\n",
      "main: quantize time = 208272.59 ms\n",
      "main:    total time = 208272.59 ms\n"
     ]
    }
   ],
   "source": [
    "# # Run the code if you would like to convert and quantize models by yourself\n",
    "# # Install Python dependencies\n",
    "# !python3 -m pip install -r requirements.txt\n",
    "\n",
    "# # Convert the HF models to ggml FP16 format (High RAM requirement!)\n",
    "# !python3 convert-hf-to-gguf.py models/Llama-3-8B-GGUF/Meta-Llama-3-8B/ --outfile models/8B-v3/ggml-model-f16.gguf --outtype f16\n",
    "!python3 convert-hf-to-gguf.py models/Llama-3-70B-GGUF/Meta-Llama-3-70B/ --outfile models/70B-v3/ggml-model-f16.gguf --outtype f16\n",
    "\n",
    "# # Quantize the model to 4-bits (using Q4_K_M method)\n",
    "# !./quantize models/8B-v3/ggml-model-f16.gguf models/8B-v3/ggml-model-Q4_K_M.gguf Q4_K_M\n",
    "!./quantize models/70B-v3/ggml-model-f16.gguf models/70B-v3/ggml-model-Q4_K_M.gguf Q4_K_M"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "30466fa3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log start\n",
      "main: build = 2796 (b3a995b4)\n",
      "main: built with Apple clang version 15.0.0 (clang-1500.3.9.4) for arm64-apple-darwin23.4.0\n",
      "main: seed  = 0\n",
      "llama_model_loader: loaded meta data with 21 key-value pairs and 291 tensors from ./models/8B-v3/ggml-model-Q4_K_M.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = Meta-Llama-3-8B\n",
      "llama_model_loader: - kv   2:                          llama.block_count u32              = 32\n",
      "llama_model_loader: - kv   3:                       llama.context_length u32              = 8192\n",
      "llama_model_loader: - kv   4:                     llama.embedding_length u32              = 4096\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 14336\n",
      "llama_model_loader: - kv   6:                 llama.attention.head_count u32              = 32\n",
      "llama_model_loader: - kv   7:              llama.attention.head_count_kv u32              = 8\n",
      "llama_model_loader: - kv   8:                       llama.rope.freq_base f32              = 500000.000000\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 15\n",
      "llama_model_loader: - kv  11:                           llama.vocab_size u32              = 128256\n",
      "llama_model_loader: - kv  12:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv  13:                       tokenizer.ggml.model str              = gpt2\n",
      "llama_model_loader: - kv  14:                         tokenizer.ggml.pre str              = llama-bpe\n",
      "llama_model_loader: - kv  15:                      tokenizer.ggml.tokens arr[str,128256]  = [\"!\", \"\\\"\", \"#\", \"$\", \"%\", \"&\", \"'\", ...\n",
      "llama_model_loader: - kv  16:                  tokenizer.ggml.token_type arr[i32,128256]  = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...\n",
      "llama_model_loader: - kv  17:                      tokenizer.ggml.merges arr[str,280147]  = [\"Ġ Ġ\", \"Ġ ĠĠĠ\", \"ĠĠ ĠĠ\", \"...\n",
      "llama_model_loader: - kv  18:                tokenizer.ggml.bos_token_id u32              = 128000\n",
      "llama_model_loader: - kv  19:                tokenizer.ggml.eos_token_id u32              = 128001\n",
      "llama_model_loader: - kv  20:               general.quantization_version u32              = 2\n",
      "llama_model_loader: - type  f32:   65 tensors\n",
      "llama_model_loader: - type q4_K:  193 tensors\n",
      "llama_model_loader: - type q6_K:   33 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 256/128256 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = BPE\n",
      "llm_load_print_meta: n_vocab          = 128256\n",
      "llm_load_print_meta: n_merges         = 280147\n",
      "llm_load_print_meta: n_ctx_train      = 8192\n",
      "llm_load_print_meta: n_embd           = 4096\n",
      "llm_load_print_meta: n_head           = 32\n",
      "llm_load_print_meta: n_head_kv        = 8\n",
      "llm_load_print_meta: n_layer          = 32\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_embd_head_k    = 128\n",
      "llm_load_print_meta: n_embd_head_v    = 128\n",
      "llm_load_print_meta: n_gqa            = 4\n",
      "llm_load_print_meta: n_embd_k_gqa     = 1024\n",
      "llm_load_print_meta: n_embd_v_gqa     = 1024\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: f_logit_scale    = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 14336\n",
      "llm_load_print_meta: n_expert         = 0\n",
      "llm_load_print_meta: n_expert_used    = 0\n",
      "llm_load_print_meta: causal attn      = 1\n",
      "llm_load_print_meta: pooling type     = 0\n",
      "llm_load_print_meta: rope type        = 0\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 500000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_yarn_orig_ctx  = 8192\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: ssm_d_conv       = 0\n",
      "llm_load_print_meta: ssm_d_inner      = 0\n",
      "llm_load_print_meta: ssm_d_state      = 0\n",
      "llm_load_print_meta: ssm_dt_rank      = 0\n",
      "llm_load_print_meta: model type       = 8B\n",
      "llm_load_print_meta: model ftype      = Q4_K - Medium\n",
      "llm_load_print_meta: model params     = 8.03 B\n",
      "llm_load_print_meta: model size       = 4.58 GiB (4.89 BPW) \n",
      "llm_load_print_meta: general.name     = Meta-Llama-3-8B\n",
      "llm_load_print_meta: BOS token        = 128000 '<|begin_of_text|>'\n",
      "llm_load_print_meta: EOS token        = 128001 '<|end_of_text|>'\n",
      "llm_load_print_meta: LF token         = 128 'Ä'\n",
      "llm_load_print_meta: EOT token        = 128009 '<|eot_id|>'\n",
      "llm_load_tensors: ggml ctx size =    0.30 MiB\n",
      "ggml_backend_metal_log_allocated_size: allocated buffer, size =  4403.50 MiB, ( 4403.56 / 147456.00)\n",
      "llm_load_tensors: offloading 32 repeating layers to GPU\n",
      "llm_load_tensors: offloading non-repeating layers to GPU\n",
      "llm_load_tensors: offloaded 33/33 layers to GPU\n",
      "llm_load_tensors:      Metal buffer size =  4403.50 MiB\n",
      "llm_load_tensors:        CPU buffer size =   281.81 MiB\n",
      ".......................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 8192\n",
      "llama_new_context_with_model: n_batch    = 2048\n",
      "llama_new_context_with_model: n_ubatch   = 512\n",
      "llama_new_context_with_model: flash_attn = 0\n",
      "llama_new_context_with_model: freq_base  = 500000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "ggml_metal_init: allocating\n",
      "ggml_metal_init: found device: Apple M2 Ultra\n",
      "ggml_metal_init: picking default device: Apple M2 Ultra\n",
      "ggml_metal_init: default.metallib not found, loading from source\n",
      "ggml_metal_init: GGML_METAL_PATH_RESOURCES = nil\n",
      "ggml_metal_init: loading '/Users/jack/llama.cpp/ggml-metal.metal'\n",
      "ggml_metal_init: GPU name:   Apple M2 Ultra\n",
      "ggml_metal_init: GPU family: MTLGPUFamilyApple8  (1008)\n",
      "ggml_metal_init: GPU family: MTLGPUFamilyCommon3 (3003)\n",
      "ggml_metal_init: GPU family: MTLGPUFamilyMetal3  (5001)\n",
      "ggml_metal_init: simdgroup reduction support   = true\n",
      "ggml_metal_init: simdgroup matrix mul. support = true\n",
      "ggml_metal_init: hasUnifiedMemory              = true\n",
      "ggml_metal_init: recommendedMaxWorkingSetSize  = 154618.82 MB\n",
      "llama_kv_cache_init:      Metal KV buffer size =  1024.00 MiB\n",
      "llama_new_context_with_model: KV self size  = 1024.00 MiB, K (f16):  512.00 MiB, V (f16):  512.00 MiB\n",
      "llama_new_context_with_model:        CPU  output buffer size =     0.49 MiB\n",
      "llama_new_context_with_model:      Metal compute buffer size =   560.00 MiB\n",
      "llama_new_context_with_model:        CPU compute buffer size =    24.01 MiB\n",
      "llama_new_context_with_model: graph nodes  = 1030\n",
      "llama_new_context_with_model: graph splits = 2\n",
      "\n",
      "system_info: n_threads = 16 / 24 | AVX = 0 | AVX_VNNI = 0 | AVX2 = 0 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 0 | NEON = 1 | ARM_FMA = 1 | F16C = 0 | FP16_VA = 1 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 0 | SSSE3 = 0 | VSX = 0 | MATMUL_INT8 = 0 | LLAMAFILE = 1 | \n",
      "sampling: \n",
      "\trepeat_last_n = 64, repeat_penalty = 1.100, frequency_penalty = 0.000, presence_penalty = 0.000\n",
      "\ttop_k = 40, tfs_z = 1.000, top_p = 0.950, min_p = 0.050, typical_p = 1.000, temp = 1.100\n",
      "\tmirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000\n",
      "sampling order: \n",
      "CFG -> Penalties -> top_k -> tfs_z -> typical_p -> top_p -> min_p -> temperature \n",
      "generate: n_ctx = 8192, n_batch = 2048, n_predict = 1024, n_keep = 0\n",
      "\n",
      "\n",
      "\u001b[33m<|begin_of_text|> First Citizen:\n",
      "\n",
      " Before we proceed any further, hear me speak.\n",
      "\n",
      " \n",
      "\n",
      " All:\n",
      "\n",
      " Speak, speak.\n",
      "\n",
      " \n",
      "\n",
      " First Citizen:\n",
      "\n",
      " You are all resolved rather to die than to famish?\n",
      "\n",
      " \n",
      "\n",
      " All:\n",
      "\n",
      " Resolved. resolved.\n",
      "\n",
      " \n",
      "\n",
      " First Citizen:\n",
      "\n",
      " First, you know Caius Marcius is chief enemy to the people.\n",
      "\n",
      " \n",
      "\n",
      " All:\n",
      "\n",
      " We know't, we know't.\n",
      "\n",
      " \n",
      "\n",
      " First Citizen:\n",
      "\n",
      " Let us kill him, and we'll have corn at our own price. Is't a verdict?\n",
      "\n",
      " \n",
      "\n",
      " All:\n",
      "\n",
      " No more talking on't; let it be done: away, away!\n",
      "\n",
      " \n",
      "\n",
      " Second Citizen:\n",
      "\n",
      " One word, good citizens.\n",
      "\n",
      " \n",
      "\n",
      " First Citizen:\n",
      "\n",
      " We are accounted poor citizens, the patricians good. What authority surfeits on would relieve us: if they would yield us but the superfluity,  while it were wholesome, we might guess they relieved us humanely; but they think we are too dear: the leanness that afflicts us, the object of  our misery, is as an inventory to particularise their abundance; our sufferance is a gain to them Let us revenge this with our pikes,  ere we become rakes: for the gods know I speak this in hunger for bread, not in thirst for revenge.\n",
      "\n",
      " \n",
      "\n",
      " \u001b[0m First Citizen:\n",
      "\n",
      " Do't when you will: 'tis like they'll never lift it.\n",
      "\n",
      " \n",
      "\n",
      " Second Citizen:\n",
      "\n",
      " Why should they then pretend to justice? Come, I trifle thus with them in havior of my wrath; now turn like spiders in my chest; the fury panteth. Ha! There goes the matter on my soul  --upon my head! Let not your eyes I prithe yield unto my tears; hold them back! Here they come: serve but for laughter, and yet farewel!\n",
      "\n",
      " \n",
      "\n",
      " First Citizen:\n",
      "\n",
      " Are these your shrewd men of the city.\n",
      "\n",
      " \n",
      "\n",
      " Second Citizen:\n",
      "\n",
      " Ha!\n",
      "\n",
      " \n",
      "\n",
      " All:\n",
      "\n",
      " What are their debts? what cannot be done to help them?\n",
      "\n",
      " \n",
      "\n",
      " Third Citizen:\n",
      "\n",
      " Knives will not pluck up grasse; tis no time in anger to  use such wrough devices: lend, lend.\n",
      "\n",
      " \n",
      "\n",
      " First Citizen:\n",
      "\n",
      " This is mere prating; and the consequences are Like stormy weather at sea. When good friends, say so\n",
      "\n",
      " \n",
      "\n",
      " Second Citizen:\n",
      "\n",
      " Let them pull down Caesar's statues and destroy His altars: Melt away the gulfs, which those superfluous shewrs Which she o'erspreads? Oth'rough-powdered infamy! Quench'd in the fire of force, let all be spent, What is't to pray to anough! what, to beseeech Him\n",
      "\n",
      " \n",
      "\n",
      " Third Citizen:\n",
      "\n",
      " Come.\n",
      "\n",
      " \n",
      "\n",
      " All:\n",
      "\n",
      " The games!\n",
      "\n",
      " \n",
      "\n",
      " First Citizen:\n",
      "\n",
      " Be it as some may. The fences may be reard To keep the creatures in: yet 'tis a course  Preposterously conceived, in means too weak And things far past all use.\n",
      "\n",
      " \n",
      "\n",
      " Second Citizen:\n",
      "\n",
      " Rescue thyself; whilst thou mayst have no need.\n",
      "\n",
      " \n",
      "\n",
      " Third Citizen:\n",
      "\n",
      " See they breathe! they move! but hear 'em not hark!\n",
      "\n",
      " \n",
      "\n",
      " All:\n",
      "\n",
      " Peace! peace!\n",
      "\n",
      " \n",
      "\n",
      " First Citizen:\n",
      "\n",
      " We'll overhear them.  --A plentifull city, I think was this Rome: as, to shine for prescience; take it for mine, A beseeming ceremony of late days; Here's Buckingham importun'd, Caes'ars uncle.  See how our Jove defies the light! Nay, but hear me.\n",
      "\n",
      " \n",
      "\n",
      " Second Citizen:\n",
      "\n",
      " Pray you, let's go see them.  --Is Caesar sick?  we have heard, oh 'tis a quarrel: Sound forth nobility. Come; see that here Is spread for these but stout men.\n",
      "\n",
      " \n",
      "\n",
      " Third Citizen:\n",
      "\n",
      " Here's the way to lay 'em in the field still more proudly than they are awake for.\n",
      "\n",
      " \n",
      "\n",
      " Second Citizen:\n",
      "\n",
      " Good faith this light pretends with him in the Cabalent, he hath rid their discretions being but rude and wanton.  --Away, away!\n",
      "\n",
      " \n",
      "\n",
      " First Citizen:\n",
      "\n",
      " Come on; it seems some excellent nectar have they tasted, and that have prov'd their vices common common to those of their houses.\n",
      "\n",
      " \n",
      "\n",
      " All:\n",
      "\n",
      " Here's goodly lordship: Prythee no more prating!  Enter BRUTUS\n",
      "\n",
      " \n",
      "\n",
      " Brutus:\n",
      "\n",
      " Stand close, my Lord and let us for a while turn our silent eyes from Mars to these Shepherds that would follow him unto the field; where by Jove's will we shall show ourselves as we are.\n",
      "\n",
      " \n",
      "\n",
      " Second Citizen:\n",
      "\n",
      " They do not go deep enough, they know not what  is to incurr himselfe a murder; for Caesar must be kill'd before we can begin afresh. Our course will seem too bloody, shoulde he die cutt him out in little love-pieces and then devour him. I think no man worth another's study: And yet by Jove, I have an over-awing spirit, that does at onetime rid his nature of his faith so that I did never dream on't.\n",
      "\n",
      " \n",
      "\n",
      " First Citizen:\n",
      "\n",
      " You say you are a heathen, will you not be christen'd and have true religion amongst us?\n",
      "\n",
      " \n",
      "\n",
      " Second Citizen:\n",
      "\n",
      " The fire from hence is easily blown to other men: if it cannot be kill'd until I do expire; I will kill 'tis necessary.\n",
      "\n",
      " \n",
      "\n",
      " First Citizen:\n",
      "\n",
      " Do so, I'll this belief of yours. If Caesar get him a new fortune, he shall not love the people the more for Caesars brother's death, than that the folks of Rome are any thing jealous of it: the one and fiftie in his heart is seated, and like to root out many a three-pearced tyrant.\n",
      "\n",
      " \n",
      "\n",
      " Brutus:\n",
      "\n",
      " Peace! do not speak to me, e'e; enuff's enough: It hath made me almost stiffe: Come gentlemen, be patient: altho Caesar may have more capacities and qualities than a weasell hath, it will be hid under a howlet's visage, never dare I draw my sword.\n",
      "\n",
      " \n",
      "\n",
      " Second Citizen:\n",
      "\n",
      " But gentlemen our country, sickness will not admit the med'cine but of noble truthe. It is not yet so\n",
      "llama_print_timings:        load time =    5265.38 ms\n",
      "llama_print_timings:      sample time =     242.31 ms /  1024 runs   (    0.24 ms per token,  4226.04 tokens per second)\n",
      "llama_print_timings: prompt eval time =     286.54 ms /   258 tokens (    1.11 ms per token,   900.41 tokens per second)\n",
      "llama_print_timings:        eval time =   14017.48 ms /  1023 runs   (   13.70 ms per token,    72.98 tokens per second)\n",
      "llama_print_timings:       total time =   14858.89 ms /  1281 tokens\n",
      "ggml_metal_free: deallocating\n",
      "Log end\n"
     ]
    }
   ],
   "source": [
    "# Start inference on a gguf model (-h to show all options)\n",
    "!./main -ngl 10000 -m ./models/8B-v3/ggml-model-Q4_K_M.gguf --color --temp 1.1 --repeat_penalty 1.1 -c 0 -n 1024 -e -s 0 -p \"\"\"\\\n",
    "First Citizen:\\n\\n\\\n",
    "Before we proceed any further, hear me speak.\\n\\n\\\n",
    "\\n\\n\\\n",
    "All:\\n\\n\\\n",
    "Speak, speak.\\n\\n\\\n",
    "\\n\\n\\\n",
    "First Citizen:\\n\\n\\\n",
    "You are all resolved rather to die than to famish?\\n\\n\\\n",
    "\\n\\n\\\n",
    "All:\\n\\n\\\n",
    "Resolved. resolved.\\n\\n\\\n",
    "\\n\\n\\\n",
    "First Citizen:\\n\\n\\\n",
    "First, you know Caius Marcius is chief enemy to the people.\\n\\n\\\n",
    "\\n\\n\\\n",
    "All:\\n\\n\\\n",
    "We know't, we know't.\\n\\n\\\n",
    "\\n\\n\\\n",
    "First Citizen:\\n\\n\\\n",
    "Let us kill him, and we'll have corn at our own price. Is't a verdict?\\n\\n\\\n",
    "\\n\\n\\\n",
    "All:\\n\\n\\\n",
    "No more talking on't; let it be done: away, away!\\n\\n\\\n",
    "\\n\\n\\\n",
    "Second Citizen:\\n\\n\\\n",
    "One word, good citizens.\\n\\n\\\n",
    "\\n\\n\\\n",
    "First Citizen:\\n\\n\\\n",
    "We are accounted poor citizens, the patricians good. What authority surfeits on would relieve us: if they would yield us but the superfluity, \\\n",
    "while it were wholesome, we might guess they relieved us humanely; but they think we are too dear: the leanness that afflicts us, the object of \\\n",
    "our misery, is as an inventory to particularise their abundance; our sufferance is a gain to them Let us revenge this with our pikes, \\\n",
    "ere we become rakes: for the gods know I speak this in hunger for bread, not in thirst for revenge.\\n\\n\\\n",
    "\\n\\n\\\n",
    "\"\"\"\n",
    "\n",
    "# # Chat template for Termianl (Use the instruction-tuned models to better follow the template)\n",
    "# !./main -ngl 10000 -m ./models/8B-v3-instruct/ggml-model-Q4_K_M.gguf --color -c 0 -n -2 -e -s 0 --mirostat 2 -i --no-display-prompt --keep -1 \\\n",
    "# -r '<|eot_id|>' -p '<|begin_of_text|><|start_header_id|>system<|end_header_id|>\\n\\nYou are a helpful assistant.<|eot_id|><|start_header_id|>user<|end_header_id|>\\n\\nHi!<|eot_id|><|start_header_id|>assistant<|end_header_id|>\\n\\n' \\\n",
    "# --in-prefix '<|start_header_id|>user<|end_header_id|>\\n\\n' --in-suffix '<|eot_id|><|start_header_id|>assistant<|end_header_id|>\\n\\n'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "17c21608",
   "metadata": {},
   "source": [
    "## Benchmarks"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2b6aee37",
   "metadata": {},
   "source": [
    "### 8B Q4_K_M"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "996ee79e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| model                          |       size |     params | backend    | ngl | test       |              t/s |\n",
      "| ------------------------------ | ---------: | ---------: | ---------- | --: | ---------- | ---------------: |\n",
      "| llama 8B Q4_K - Medium         |   4.58 GiB |     8.03 B | Metal      |  99 | pp 512     |   994.04 ± 21.80 |\n",
      "| llama 8B Q4_K - Medium         |   4.58 GiB |     8.03 B | Metal      |  99 | pp 1024    |   1023.89 ± 7.60 |\n",
      "| llama 8B Q4_K - Medium         |   4.58 GiB |     8.03 B | Metal      |  99 | pp 2048    |   1015.35 ± 1.00 |\n",
      "| llama 8B Q4_K - Medium         |   4.58 GiB |     8.03 B | Metal      |  99 | pp 4096    |    979.47 ± 0.77 |\n",
      "| llama 8B Q4_K - Medium         |   4.58 GiB |     8.03 B | Metal      |  99 | pp 8192    |    913.55 ± 1.95 |\n",
      "| llama 8B Q4_K - Medium         |   4.58 GiB |     8.03 B | Metal      |  99 | tg 512     |     78.81 ± 0.02 |\n",
      "| llama 8B Q4_K - Medium         |   4.58 GiB |     8.03 B | Metal      |  99 | tg 1024    |     76.28 ± 0.07 |\n",
      "| llama 8B Q4_K - Medium         |   4.58 GiB |     8.03 B | Metal      |  99 | tg 2048    |     72.23 ± 0.04 |\n",
      "| llama 8B Q4_K - Medium         |   4.58 GiB |     8.03 B | Metal      |  99 | tg 4096    |     64.58 ± 0.04 |\n",
      "| llama 8B Q4_K - Medium         |   4.58 GiB |     8.03 B | Metal      |  99 | tg 8192    |     54.13 ± 0.09 |\n",
      "\n",
      "build: b3a995b4 (2796)\n"
     ]
    }
   ],
   "source": [
    "!./llama-bench -p 512,1024,2048,4096,8192 -n 512,1024,2048,4096,8192 -m ./models/8B-v3/ggml-model-Q4_K_M.gguf"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0f07d6cd",
   "metadata": {},
   "source": [
    "### 8B F16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "cb0b80b3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| model                          |       size |     params | backend    | ngl | test       |              t/s |\n",
      "| ------------------------------ | ---------: | ---------: | ---------- | --: | ---------- | ---------------: |\n",
      "| llama 8B F16                   |  14.96 GiB |     8.03 B | Metal      |  99 | pp 512     |  1175.40 ± 16.77 |\n",
      "| llama 8B F16                   |  14.96 GiB |     8.03 B | Metal      |  99 | pp 1024    |  1202.74 ± 19.43 |\n",
      "| llama 8B F16                   |  14.96 GiB |     8.03 B | Metal      |  99 | pp 2048    |  1220.05 ± 10.89 |\n",
      "| llama 8B F16                   |  14.96 GiB |     8.03 B | Metal      |  99 | pp 4096    |   1194.21 ± 3.92 |\n",
      "| llama 8B F16                   |  14.96 GiB |     8.03 B | Metal      |  99 | pp 8192    |   1103.44 ± 2.32 |\n",
      "| llama 8B F16                   |  14.96 GiB |     8.03 B | Metal      |  99 | tg 512     |     36.90 ± 0.10 |\n",
      "| llama 8B F16                   |  14.96 GiB |     8.03 B | Metal      |  99 | tg 1024    |     36.25 ± 0.24 |\n",
      "| llama 8B F16                   |  14.96 GiB |     8.03 B | Metal      |  99 | tg 2048    |     34.83 ± 0.65 |\n",
      "| llama 8B F16                   |  14.96 GiB |     8.03 B | Metal      |  99 | tg 4096    |     33.67 ± 0.32 |\n",
      "| llama 8B F16                   |  14.96 GiB |     8.03 B | Metal      |  99 | tg 8192    |     30.68 ± 0.08 |\n",
      "\n",
      "build: b3a995b4 (2796)\n"
     ]
    }
   ],
   "source": [
    "!./llama-bench -p 512,1024,2048,4096,8192 -n 512,1024,2048,4096,8192 -m ./models/8B-v3/ggml-model-f16.gguf"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "72d34884",
   "metadata": {},
   "source": [
    "### 70B Q4_K_M"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "8a4858d0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| model                          |       size |     params | backend    | ngl | test       |              t/s |\n",
      "| ------------------------------ | ---------: | ---------: | ---------- | --: | ---------- | ---------------: |\n",
      "| llama 70B Q4_K - Medium        |  39.59 GiB |    70.55 B | Metal      |  99 | pp 512     |    118.79 ± 0.25 |\n",
      "| llama 70B Q4_K - Medium        |  39.59 GiB |    70.55 B | Metal      |  99 | pp 1024    |    117.76 ± 0.19 |\n",
      "| llama 70B Q4_K - Medium        |  39.59 GiB |    70.55 B | Metal      |  99 | pp 2048    |    116.34 ± 0.06 |\n",
      "| llama 70B Q4_K - Medium        |  39.59 GiB |    70.55 B | Metal      |  99 | pp 4096    |    109.53 ± 3.41 |\n",
      "| llama 70B Q4_K - Medium        |  39.59 GiB |    70.55 B | Metal      |  99 | pp 8192    |    108.57 ± 0.09 |\n",
      "| llama 70B Q4_K - Medium        |  39.59 GiB |    70.55 B | Metal      |  99 | tg 512     |     12.48 ± 0.00 |\n",
      "| llama 70B Q4_K - Medium        |  39.59 GiB |    70.55 B | Metal      |  99 | tg 1024    |     12.13 ± 0.15 |\n",
      "| llama 70B Q4_K - Medium        |  39.59 GiB |    70.55 B | Metal      |  99 | tg 2048    |     11.69 ± 0.08 |\n",
      "| llama 70B Q4_K - Medium        |  39.59 GiB |    70.55 B | Metal      |  99 | tg 4096    |     10.75 ± 0.08 |\n",
      "| llama 70B Q4_K - Medium        |  39.59 GiB |    70.55 B | Metal      |  99 | tg 8192    |      9.34 ± 0.03 |\n",
      "\n",
      "build: b3a995b4 (2796)\n"
     ]
    }
   ],
   "source": [
    "!./llama-bench -p 512,1024,2048,4096,8192 -n 512,1024,2048,4096,8192 -m ./models/70B-v3/ggml-model-Q4_K_M.gguf"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f09ce3e8",
   "metadata": {},
   "source": [
    "### 70B F16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "273f323b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| model                          |       size |     params | backend    | ngl | test       |              t/s |\n",
      "| ------------------------------ | ---------: | ---------: | ---------- | --: | ---------- | ---------------: |\n",
      "| llama 70B F16                  | 131.42 GiB |    70.55 B | Metal      |  99 | pp 512     |    147.58 ± 0.70 |\n",
      "| llama 70B F16                  | 131.42 GiB |    70.55 B | Metal      |  99 | pp 1024    |    145.82 ± 0.80 |\n",
      "| llama 70B F16                  | 131.42 GiB |    70.55 B | Metal      |  99 | pp 2048    |    144.13 ± 0.31 |\n",
      "| llama 70B F16                  | 131.42 GiB |    70.55 B | Metal      |  99 | pp 4096    |    133.75 ± 5.21 |\n",
      "| llama 70B F16                  | 131.42 GiB |    70.55 B | Metal      |  99 | pp 8192    |    135.15 ± 0.22 |\n",
      "| llama 70B F16                  | 131.42 GiB |    70.55 B | Metal      |  99 | tg 512     |      4.76 ± 0.01 |\n",
      "| llama 70B F16                  | 131.42 GiB |    70.55 B | Metal      |  99 | tg 1024    |      4.71 ± 0.00 |\n",
      "| llama 70B F16                  | 131.42 GiB |    70.55 B | Metal      |  99 | tg 2048    |      4.62 ± 0.03 |\n",
      "| llama 70B F16                  | 131.42 GiB |    70.55 B | Metal      |  99 | tg 4096    |      4.48 ± 0.08 |\n",
      "| llama 70B F16                  | 131.42 GiB |    70.55 B | Metal      |  99 | tg 8192    |      4.23 ± 0.02 |\n",
      "\n",
      "build: b3a995b4 (2796)\n"
     ]
    }
   ],
   "source": [
    "!./llama-bench -p 512,1024,2048,4096,8192 -n 512,1024,2048,4096,8192 -m ./models/70B-v3/ggml-model-f16.gguf"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
